# -*- coding: utf-8 -*-
import scrapy

import json
import datetime
from copy import deepcopy, copy
from scrapy_redis.spiders import RedisSpider
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem


# class FoshanSpider(scrapy.Spider):
class FoshanSpider(RedisSpider):
    name = 'foshan'
    allowed_domains = ['fsfc.fszj.foshan.gov.cn']
    redis_key = 'foshan'

    # start_urls = ['http://fsfc.fsjw.gov.cn/search/index.do?ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE']  # 已过期
    # start_urls = ['http://fsfc.fszj.foshan.gov.cn/search/index.do?ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE']  # 升级为https协议
    # start_urls = ['https://fsfc.fszj.foshan.gov.cn/search/index.do?ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE']

    def parse(self, response):
        # 获取当前页楼盘页表分组
        dl_li = response.xpath('//dl')
        for dl in dl_li:
            item_eg = FdcEstateGuidItem()
            item_eg['projectId'] = dl.xpath('.//h3/a/@value').extract_first()
            item_eg['projectName'] = dl.xpath('.//h3/a/text()').extract_first()

            soldTotalRoomNo = dl.xpath('.//p[last()]/em[1]/text()').extract()
            item_eg['soldTotalRoomNo'] = soldTotalRoomNo[0] if (
                    len(soldTotalRoomNo) > 0 and soldTotalRoomNo[0] != '') else None

            unsoldTotalRoomNo = dl.xpath('.//p[last()]/em[2]/text()').extract()
            item_eg['unsoldTotalRoomNo'] = unsoldTotalRoomNo[0] if (
                    len(unsoldTotalRoomNo) > 0 and unsoldTotalRoomNo[0] != '') else None

            # 楼盘url
            # estate_url_temp = 'http://fsfc.fsjw.gov.cn/hpms_project/roomView.jhtml?id={}'  # 已过期
            # estate_url_temp = 'http://fsfc.fszj.foshan.gov.cn/hpms_project/roomView.jhtml?id={}'  # 升级为https协议
            estate_url_temp = 'https://fsfc.fszj.foshan.gov.cn/hpms_project/roomView.jhtml?id={}'
            item_eg['estateUrl'] = estate_url_temp.format(int(item_eg['projectId']))
            yield scrapy.Request(
                item_eg['estateUrl'],
                callback=self.parse_estate,
                meta={"item_eg": deepcopy(item_eg)},
            )

        # 翻页
        total_num = int(response.xpath("//div[@class='tab-tit']//em/text()").extract_first())
        total_page_num = total_num // 10 + 1 if total_num % 10 != 0 else total_num // 10
        # next_page_url_temp = 'http://fsfc.fsjw.gov.cn/search/index.do?sw=&dn=&hx=&lx=&mj=&jg=&ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE&p={}'  # 已过期
        # next_page_url_temp = 'http://fsfc.fszj.foshan.gov.cn/search/index.do?sw=&dn=&hx=&lx=&mj=&jg=&ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE&p={}'  # 升级为https协议
        next_page_url_temp = 'https://fsfc.fszj.foshan.gov.cn/search/index.do?sw=&dn=&hx=&lx=&mj=&jg=&ys=0&od=-SORT;-STIME;-LASTMODIFYTIME;RELEVANCE&p={}'
        next_page_url_li = [next_page_url_temp.format(i) for i in range(2, total_page_num + 1)]
        for next_url in next_page_url_li:
            yield scrapy.Request(
                next_url,
                callback=self.parse,
            )

    def parse_estate(self, response):
        """
        获取楼盘信息
        :param response:
        :return:
        """
        item_eg = copy(response.meta["item_eg"])
        item_eg['estateAddress'] = response.xpath("//div[@class='wzjs-box']//table//tr[2]//span/@title").extract_first()
        item_eg['developerName'] = response.xpath(
            "//div[@class='wzjs-box']//table//tr[3]/td[1]//span/@title").extract_first()
        item_eg['districtName'] = response.xpath("//div[@class='wzjs-box']//table//tr[3]/td[2]/text()").extract_first()

        propertyCompany = response.xpath("//div[@class='wzjs-box']//table//tr[4]/td[1]/span/@title").extract()
        item_eg['propertyCompany'] = propertyCompany[0] if (
                len(propertyCompany) > 0 and propertyCompany[0] != '') else None

        propertyManagementFee = response.xpath("//div[@class='wzjs-box']//table//tr[4]/td[2]/span/@title").extract()
        item_eg['propertyManagementFee'] = propertyManagementFee[0] if (
                len(propertyManagementFee) > 0 and propertyManagementFee[0] != '') else None

        estateTotalArea = response.xpath("//div[@class='wzjs-box']//table//tr[5]/td[1]/text()").extract()
        item_eg['estateTotalArea'] = estateTotalArea[0] if (
                len(estateTotalArea) > 0 and estateTotalArea[0] != '') else None

        floorAreaRatio = response.xpath("//div[@class='wzjs-box']//table//tr[5]/td[2]/span/@title").extract()
        item_eg['floorAreaRatio'] = floorAreaRatio[0] if (
                len(floorAreaRatio) > 0 and floorAreaRatio[0] != '') else None

        qualificationCertificateNo = response.xpath("//div[@class='wzjs-box']//table//tr[6]/td[1]/text()").extract()
        item_eg['qualificationCertificateNo'] = qualificationCertificateNo[0] if (
                len(qualificationCertificateNo) > 0 and qualificationCertificateNo[0] != '') else None

        qualificationGrade = response.xpath("//div[@class='wzjs-box']//table//tr[6]/td[2]/text()").extract()
        item_eg['qualificationGrade'] = qualificationGrade[0] if (
                len(qualificationGrade) > 0 and qualificationGrade[0] != '') else None

        avgPrice = response.xpath("//div[@class='wzjs-box']//table//tr[7]/td[1]/em/text()").extract()
        item_eg['avgPrice'] = avgPrice[0] if (
                len(avgPrice) > 0 and avgPrice[0] != '') else None

        salesTel = response.xpath("//div[@class='wzjs-box']//table//tr[7]/td[2]/span/@title").extract()
        item_eg['salesTel'] = salesTel[0] if (
                len(salesTel) > 0 and salesTel[0] != '') else None

        # 构造预售许可证请求
        # cert_li_url_temp = 'http://fsfc.fsjw.gov.cn/hpms_project/ysxkz.jhtml?id={}&zmc='  # 已过期
        # cert_li_url_temp = 'http://fsfc.fszj.foshan.gov.cn/hpms_project/ysxkz.jhtml?id={}&zmc='  # 升级为https协议
        cert_li_url_temp = 'https://fsfc.fszj.foshan.gov.cn/hpms_project/ysxkz.jhtml?id={}&zmc='
        yield scrapy.Request(
            cert_li_url_temp.format(int(item_eg['projectId'])),
            callback=self.parse_cert_li,
            meta={"item_eg": deepcopy(item_eg)}
        )

        # 获取楼栋分组
        a_li = response.xpath("//a[contains(@href,'roomView.jhtml')]")
        for a in a_li:
            item_bd = FdcBuildingItem()
            item_bd['projectName'] = item_eg['projectName']
            item_bd['districtName'] = item_eg['districtName']
            item_bd['estateAddress'] = item_eg['estateAddress']
            item_bd['developerName'] = item_eg['developerName']
            item_bd['blockName'] = a.xpath('./text()').extract_first()
            item_bd['buildingId'] = a.xpath('./@id').extract_first()
            # 楼栋url
            # building_url_temp = 'http://fsfc.fsjw.gov.cn/hpms_project/roomtj.jhtml?id={}'  # 已过期
            # building_url_temp = 'http://fsfc.fszj.foshan.gov.cn/hpms_project/roomtj.jhtml?id={}'  # 升级为https协议
            building_url_temp = 'https://fsfc.fszj.foshan.gov.cn/hpms_project/roomtj.jhtml?id={}'
            item_bd['buildingUrl'] = building_url_temp.format(int(item_bd['buildingId']))
            yield scrapy.Request(
                item_bd['buildingUrl'],
                callback=self.parse_building,
                meta={"item_bd": deepcopy(item_bd)}
            )

            item_rm = FdcRoomItem()
            item_rm['districtName'] = item_eg['districtName']
            item_rm['projectName'] = item_eg['projectName']
            item_rm['developerName'] = item_eg['developerName']
            item_rm['blockName'] = item_bd['blockName']
            item_rm['buildingId'] = item_bd['buildingId']
            # 房屋列表url
            # room_li_url_temp = 'http://fsfc.fsjw.gov.cn/hpms_project/room.jhtml?id={}'  # 已过期
            # room_li_url_temp = 'http://fsfc.fszj.foshan.gov.cn/hpms_project/room.jhtml?id={}'  # 升级为https协议
            room_li_url_temp = 'https://fsfc.fszj.foshan.gov.cn/hpms_project/room.jhtml?id={}'
            yield scrapy.Request(
                room_li_url_temp.format(int(item_bd['buildingId'])),
                callback=self.parse_room_li,
                meta={"item_rm": deepcopy(item_rm)}
            )

    def parse_cert_li(self, response):
        """
        获取预售许可证列表信息
        :param response:
        :return:
        """
        item_eg = copy(response.meta["item_eg"])
        # 获取预售许可证列表分组
        # cert_url_temp = 'http://fsfc.fsjw.gov.cn/hpms_project/ysxkzxx.jhtml?xkzh={}'  # 已过期
        # cert_url_temp = 'http://fsfc.fszj.foshan.gov.cn/hpms_project/ysxkzxx.jhtml?xkzh={}'  # 升级为https协议
        cert_url_temp = 'https://fsfc.fszj.foshan.gov.cn/hpms_project/ysxkzxx.jhtml?xkzh={}'
        option_li = response.xpath('//select/option')
        for option in option_li:
            item_eg['preSalePermit'] = option.xpath('./text()').extract_first()
            cert_id = option.xpath('./@value').extract_first()
            yield scrapy.Request(
                cert_url_temp.format(int(cert_id)),
                callback=self.parse_cert,
                meta={"item_eg": deepcopy(item_eg)}
            )

    def parse_cert(self, response):
        """
        获取预售许可证信息
        :param response:
        :return:
        """
        item_eg = copy(response.meta["item_eg"])
        cert_dict = json.loads(response.body.decode())
        item_eg['certDate'] = self.transfer_date(cert_dict['fzrq']) if cert_dict['fzrq'] else None
        item_eg['certStartDate'] = self.transfer_date(cert_dict['yxqx1']) if cert_dict['yxqx1'] else None
        item_eg['certExpiryDate'] = self.transfer_date(cert_dict['yxqx2']) if cert_dict['yxqx2'] else None
        item_eg['preSaleRoomNo'] = cert_dict['yszts'] if cert_dict['yszts'] else None
        item_eg['preSaleArea'] = cert_dict['yszmj'] if cert_dict['yszmj'] else None
        item_eg['preSaleBlockName'] = cert_dict['jdZh1'] if cert_dict['jdZh1'] else None
        item_eg['CA'] = cert_dict['fzjg'] if cert_dict['fzjg'] else None
        item_eg['preSaleDepositBank'] = cert_dict['yszjkhyh'] if cert_dict['yszjkhyh'] else None
        item_eg['preSaleAccountNum'] = cert_dict['yszjzh'] if cert_dict['yszjzh'] else None
        item_eg['commerceNum'] = cert_dict['syts'] if cert_dict['syts'] else None
        item_eg['commerceArea'] = cert_dict['symj'] if cert_dict['symj'] else None
        item_eg['residenceNum'] = cert_dict['zzts'] if cert_dict['zzts'] else None
        item_eg['residenceArea'] = cert_dict['zzmj'] if cert_dict['zzmj'] else None
        item_eg['otherNum'] = cert_dict['qtts'] if cert_dict['qtts'] else None
        item_eg['otherArea'] = cert_dict['qtmj'] if cert_dict['qtmj'] else None
        yield item_eg

    def parse_building(self, response):
        """
        获取楼栋信息
        :param response:
        :return:
        """
        item_bd = copy(response.meta["item_bd"])
        building_dict = json.loads(response.body.decode())
        item_bd['unsoldRoomNo'] = building_dict.get('wsts', None)
        item_bd['soldRoomNo'] = building_dict.get('ysts', None)
        item_bd['floorTotalNo'] = building_dict.get('zcs', None)
        yield item_bd

    def parse_room_li(self, response):
        """
        获取房间列表信息
        :param response:
        :return:
        """
        item_rm = copy(response.meta["item_rm"])
        room_li = json.loads(response.body.decode())
        for room in room_li:
            item_rm['roomNo'] = room.get('roomno', None)
            item_rm['roomId'] = room.get('fwcode', None)
            item_rm['saleStatus'] = room.get('zt', None)
            item_rm['roomUse'] = room.get('ghyt', None)
            item_rm['unitName'] = room.get('unitname', None) if room.get('unitname', None) != '1000' else None
            item_rm['roomFloor'] = room.get('floorindex', None)
            item_rm['preSalePermit'] = room.get('xkzh', None)
            item_rm['roomLocation'] = room.get('fwzl', None)
            item_rm['buildingArea'] = room.get('jzmj', None)
            item_rm['innerArea'] = room.get('tnmj', None)
            # room_url_temp = 'http://fsfc.fsjw.gov.cn/hpms_project/roomview.jhtml?id={}&xmmc='  # 已过期
            # room_url_temp = 'http://fsfc.fszj.foshan.gov.cn/hpms_project/roomview.jhtml?id={}&xmmc='  # 升级为https协议
            room_url_temp = 'https://fsfc.fszj.foshan.gov.cn/hpms_project/roomview.jhtml?id={}&xmmc='
            if item_rm['saleStatus'] == '可售':
                yield scrapy.Request(
                    room_url_temp.format(int(room['fwcode'])),
                    callback=self.parse_room,
                    meta={"item_rm": deepcopy(item_rm)}
                )
            else:
                item_rm['roomTotalPrice'] = None
                yield item_rm

    def parse_room(self, response):
        """
        获取房间信息
        :param response:
        :return:
        """
        item_rm = copy(response.meta["item_rm"])
        roomTotalPrice = response.xpath("//table[@class='table01']//tr[10]/td[4]/text()").extract()
        item_rm['roomTotalPrice'] = roomTotalPrice[0].strip() if (
                len(roomTotalPrice) > 0 and roomTotalPrice[0].strip() != str(0) and roomTotalPrice[
            0].strip() != '*****') else None
        yield item_rm

    def transfer_date(self, stamp):
        """
        将时间戳转换为date
        :param stamp:
        :return:
        """
        date_obj = datetime.datetime.fromtimestamp(int(stamp) / 1000, None)
        date_str = date_obj.strftime("%Y-%m-%d")
        return date_str
